from selenium import webdriver
import time
from selenium.webdriver.chrome.options import Options
import requests
import re
import MySQLdb
from MySQLdb import escape_string


class mouair(object):
    '''
    1.https://zh.airbnb.com/s/'+ location +r'/homes 拿到各个省会城市的房源id
        1.通过查找是否还有下一页确定是否往下翻页
        2.解析出各个页面的url
        3.url存储到数据库
    2.读出每一个房源的url
        1.解析出
            1.经纬度坐标 2.周市，省份，国别  3.客房信息()  4.客房title   5.客房价格  6.客房简介
            2.解析出房东id，进入房东主页面
                解析出:
                    1.用户图片url   2.用户简介
                    3.客房评价
                        检查是否还有更多内容按钮，有点击，等待一秒，往下
                        匹配出数据

    '''

    def __init__(self):
        opt = Options()
        opt.add_argument('--headless')
        self.__browser = webdriver.Chrome(chrome_options=opt)
        self.__browser.maximize_window()
        # 进到用户界面在爬
        self.user_picture_tag = r'_1oa3geg'  # 可以通过这个tag找到对应的标签和
        self.user_name = r'您好，我是.{1,16}！'  # 匹配全片的第一个 匹配出来后选[5:-1]
        self.user_introduction_tag = r'space-top-2'  # selenium 找到第一个space-top-2 然后获取其内容
        # 进到用户界面再爬
        # 通过 css_selector来获取 comment-container expandable expandable-trigger-more space-2 expanded
        # self.comment = None # 加载更多，如果load_more还存在

        # self.location = r'location=^-?((0|1?[0-7]?[0-9]?)(([.][0-9]{1,4})?)|180(([.][0]{1,4})?))$,^-?((0|[1-8]?[0-9]?)(([.][0-9]{1,4})?)|90(([.][0]{1,4})?))$&zoom'
        self.location = r'-?\d{1,3}\.\d{4,6},-?\d{1,2}\.\d{4,6}'  # 不一定能匹配出经纬度，后面再改进正则

        self.province_tag = r'_vbshb6'

        # self.room_infor = None # 客房信息采用正则表达式获取匹配 \d{1,2}间卧室  \d{1,2}张床   \d{1,2]间卫生间 最多住\d{1,2}人或者使用_16ono4k6查找出0123就是
        self.room_title_tag = r'_wo2sh4'
        self.room_price = r'\"_doc79r\"\>￥\d{1,5}'  # r'<span class="_doc79r">￥268</span>'
        self.room_introduction_tag = '_6z3til'  # 先找到加载更多，然后通过css _6z3til直接获取最后一个元素的文本

        self.db = MySQLdb.connect(host='127.0.0.1', port=3306, user='root', passwd='mysql555', db='airbnb',
                                  charset='utf8')
        self.init_db()

    def init_db(self):
        cursor = self.db.cursor()
        create_rooms_url = """create table if not exists rooms_url( id integer primary key auto_increment, room_url varchar(256) not null);"""
        cursor.execute(create_rooms_url)

        create_room_infor = """create table if not exists rooms_infor(
                            title varchar(128),
                            price integer ,
                            src varchar(256),
                            province varchar(32),
                            location varchar(64),
                            Rintro varchar(256),
                            bedroom integer ,
                            bed integer ,
                            toilet integer ,
                            max_num integer ,
                            user_id varchar(32) primary key,
                            picture_url varchar(256),
                            user_intro varchar (256),
                            user_name varchar(16));"""
        cursor.execute(create_room_infor)

        create_users_rul = """create table if not exists users_url( id integer primary key auto_increment, user_url varchar(256) not null);"""
        cursor.execute(create_users_rul)

        create_comments = """create table if not exists comments( 
                         id integer primary key auto_increment, 
                         user_id varchar(32), 
                         content text);"""
        cursor.execute(create_comments)

        cursor.close()

    def get_rooms_url(self, location=r'贵阳--贵州省--中国'):
        '''
        解析出当前页面的房源信息，并写入数据库，默认搜索 贵阳--贵州省--中国
        :param location:
        :return:
        '''
        self.__browser.get(r'https://zh.airbnb.com/s/' + location + r'/homes')
        time.sleep(2)
        next_page = self.is_exist_by_class_name('_5u96sq')
        cursor = self.db.cursor()
        while next_page:
            result = list(re.findall('content=\"zh\.airbnb\.com\/rooms\/[0-9]{6,9}', self.__browser.page_source))
            try:
                for i in result:
                    cursor.execute(r'insert into rooms_url(room_url)'
                                   r' values (%s)' % i[9:])
                self.db.commit()
            except:
                self.db.rollback()
            next_page.click()
            time.sleep(1)
            # 检查是否有下一页
            next_page = self.is_exist_by_class_name('_5u96sq')
        cursor.close()

    def get_room_informations(self, room_url=r'https://zh.airbnb.com/rooms/29859800?guests=1&adults=1'):
        '''
        读出数据库中的房源url,进入房源解析
        :param room_url:
        :return:
        '''
        self.__browser.get(room_url)
        time.sleep(1)
        cursor = self.db.cursor()
        # try:
        time.sleep(3)
        title = self.__browser.find_element_by_class_name('_wo2sh4').text
        province = self.__browser.find_element_by_css_selector('div._vbshb6').text
        temp = self.__browser.find_element_by_css_selector('button>div:nth-child(1)>img')
        location = re.findall(r'-?\d{1,3}\.\d{0,6},-?\d{1,2}\.\d{0,6}', temp.get_attribute('src'))
        temp = self.__browser.find_element_by_css_selector('div[dir="ltr"] > p._6z3til > span._16ono4k6')
        Rintro = temp.text
        temp = self.__browser.find_elements_by_css_selector('div._1thk0tsb>span._16ono4k6')
        bedroom = int(temp[0].text[0])
        bed = int(temp[1].text[0])
        toilet = int(temp[2].text[0])
        max_num = int(temp[3].text[3])
        picture_url = self.__browser.find_element_by_css_selector(
            'img._1mgxxu3[width="88"][height="88"][title][alt][src]').get_attribute('src')
        temp = re.findall('href=\"\/users\/show\/\d{4,12}', self.__browser.page_source)
        user_id = temp[-1][18:]
        temp = self.__browser.find_elements_by_css_selector('li[data-id]')
        page_count = int(temp[-1].text)

        temp = re.findall('\"_doc79r\"\>￥\d{1,5}', self.__browser.page_source)
        if temp.__len__() == 0:
            return
        else:
            price = int(temp[0][11:])

        self.__browser.get(r'https://zh.airbnb.com/users/show/' + user_id)
        time.sleep(1)

        self.comments_next_page(page_count)
        comments = self.__browser.find_elements_by_css_selector(
            'div.comment-container.expandable.expandable-trigger-more.space-2.expanded>div.expandable-content')
        if comments.__len__() == 0:
            return
        for i in range(comments.__len__() // 2):
            cursor.execute(
                'insert into comments (user_id, content) values (' + '"' + str(user_id) + '"' + ',' + '"' + str(
                    comments[i].text) + '"' + ')')
        user_name = re.findall(r'您好，我是.{1,16}！', self.__browser.page_source)[0][5:-1]
        user_intro = self.__browser.find_element_by_css_selector('div.space-top-2>p').text


        cursor.execute(r'insert into rooms_infor('
                       r'title,price,src,province,location,Rintro,bedroom,bed,toilet,max_num,user_id,picture_url,user_intro,user_name)'
                       r'values ('
                       r'%s,%d,%s,%s,%s,%s,%d,%d,%d,%d,%s,%s,%s,%s)' % (
                           self.repalace(title), price, self.repalace(room_url), self.repalace(province),
                           self.repalace(location),
                           self.repalace(Rintro), bedroom, bed, toilet, max_num, self.repalace(user_id),
                           self.repalace(picture_url), self.repalace(user_intro), self.repalace(user_name)
                       ))
        self.db.commit()
        # except Exception as e:
        #     print(e)
        # finally:
        cursor.close()

    def repalace(self, para):
        return '"' + str(para) + '"'

    def comments_next_page(self,page_count):
        load_more = self.__browser.find_elements_by_css_selector('a.load_more[href][data-prevent-default][data-page]')
        next_page = load_more[0]
        for i in range(page_count - 1):
            next_page.click()

if __name__ == '__main__':
    # browser = webdriver.Chrome()
    # browser.maximize_window()
    # # browser.get(r'https://zh.airbnb.com//users/show/194394814')
    # browser.get(r'https://zh.airbnb.com//users/show/194394814')
    # #视网速而定，等待JS请求返回数据
    # time.sleep(1)
    #
    # user_name = re.search(r'您好，我是.{1,16}！',browser.page_source)
    # user_picture_url = re.findall(r'https://z1\.muscache\.cn\/im\/pictures\/user\/.*\.jpg\?aki_policy=profile_x_medium',browser.page_source)[0]
    # user_intro = browser.find_elements_by_class_name('space-top-2')[2].text
    #
    #
    # load_more = browser.find_element_by_class_name('load_more')
    # while load_more:
    #     load_more.click()
    #     time.sleep(1)
    airbnb = mouair()
    airbnb.get_room_informations()
    pass
    # div.comment - container, expandable, expandable - trigger - more.space - 2.
    # expanded > div.expandable - content > p
